from optparse import OptionParser
import httplib
import urllib
import re
import fileinput
import sgmllib
import yaml
from HTMLParser import HTMLParser

ITEMS_PER_PAGE = 10
HEADERS = {'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.7) Gecko/20070914 Firefox/2.0.0.7'}

def main():
    parser = OptionParser()
    parser.add_option("-f", dest="file", help="import category config",
                      default="category.yandex")
    parser.add_option("-o", dest="out_dir", help="output directory",
                      default="yandex")
    (options, args) = parser.parse_args()
    for line in fileinput.input(options.file):
        parts = line.split()
        import_category(parts[0], int(parts[1]), int(parts[2]), parts[3],
                         options.out_dir)

def import_category(name, start, end, list_url_pattern, out_dir):
    current = start
    con = httplib.HTTPConnection("market.yandex.ru")
    while current < end:
        url = list_url_pattern % current
        con.request('GET', url, headers=HEADERS)
        res = con.getresponse()
        if res.status == 200:
            data = res.read()
            product_urls = set(re.compile('(?<=")/model\.xml[^"]*(?=")').findall(data))
            print 'loading urls', product_urls
            for product_url in product_urls:
                con.request('GET', product_url, HEADERS)
                res = con.getresponse()
                
        else:
            raise RuntimeError, 'returned wrong response code %s for url %s' % (res.status, url)
        current += ITEMS_PER_PAGE
    con.close()

(MODE_DEFAULT, MODE_SET_NAME, MODE_SET_GROUP_NAME, MODE_START_PROPERTY_NAME,
MODE_SET_PROPERTY_NAME, MODE_START_PROPERTY_VALUE,
MODE_SET_PROPERTY_VALUE) = range(7)

class ProductPageParser(sgmllib.SGMLParser):
    _product = None
    _mode = MODE_DEFAULT 
    _current_group = {}
    _current_prop = None
    _data = ''
    
    from htmlentitydefs import entitydefs
    
    def reset(self):
        self._product = {'name':'', 'images':[], 'data':{}}
        self._mode = MODE_DEFAULT
        self._current_group = {}
        self._current_prop = None
        _data = ''
        sgmllib.SGMLParser.reset(self)
   
    def unknown_starttag(self, tag, attrs):
        self._data = ''    
        print 'start tag'
        if self._mode == MODE_DEFAULT and tag == 'h1' and len(attrs) > 0:
            self._mode = MODE_SET_NAME
        elif (self._mode == MODE_DEFAULT or self._mode == MODE_START_PROPERTY_NAME) and tag == 'td' and attrs == [('colspan', '2'), ('class', 'title')]:
            self._mode = MODE_SET_GROUP_NAME
        elif self._mode == MODE_START_PROPERTY_NAME and tag == 'td' and attrs == [('class', 'label')]:
            self._mode = MODE_SET_PROPERTY_NAME
        elif self._mode == MODE_START_PROPERTY_VALUE and tag == 'td':
            self._mode = MODE_SET_PROPERTY_VALUE
        elif tag == 'a' and len(attrs) > 0 and attrs[0][1].startswith('http://data.yandex.ru/i?ctype=1'):
            self._product['images'].append(attrs[0][1])
                    
    
    def handle_data(self, data):
        if data:
            self._data = self._data + data

    def handle_entityref(self, name):
        if self.entitydefs.has_key(name): 
            self._data = self._data + self.entitydefs[name]
    
    def unknown_endtag(self, tag):
        print 'end tag'
        self._data = self._data.strip()
        if self._mode == MODE_SET_NAME and tag == 'h1':
            self._mode = MODE_DEFAULT
            self._product['name'] = self._data
        elif self._mode == MODE_SET_GROUP_NAME and tag == 'td':
            self._current_group = {}
            self._product['data'][self._data] = self._current_group
            self._mode = MODE_START_PROPERTY_NAME  
        if self._mode == MODE_SET_PROPERTY_NAME and tag == 'td':
            self._mode = MODE_START_PROPERTY_VALUE
            self._current_prop = self._data
        if self._mode == MODE_SET_PROPERTY_VALUE and tag == 'td':
            self._mode = MODE_START_PROPERTY_NAME
            self._current_group[self._current_prop] = self._data
   
    def get_result(self):
        return self._product

class ProductPageArrayParser(sgmllib.SGMLParser):
    _product = None
    _mode = MODE_DEFAULT 
    _current_group = None
    _current_prop = None
    _data = ''
    
    from htmlentitydefs import entitydefs
    
    def reset(self):
        self._product = ['', [], []]
        self._mode = MODE_DEFAULT
        self._current_group = []
        self._current_prop = None
        _data = ''
        sgmllib.SGMLParser.reset(self)
   
    def unknown_starttag(self, tag, attrs):
        self._data = ''    
        print 'start tag'
        if self._mode == MODE_DEFAULT and tag == 'h1' and len(attrs) > 0:
            self._mode = MODE_SET_NAME
        elif (self._mode == MODE_DEFAULT or self._mode == MODE_START_PROPERTY_NAME) and tag == 'td' and attrs == [('colspan', '2'), ('class', 'title')]:
            self._mode = MODE_SET_GROUP_NAME
        elif self._mode == MODE_START_PROPERTY_NAME and tag == 'td' and attrs == [('class', 'label')]:
            self._mode = MODE_SET_PROPERTY_NAME
        elif self._mode == MODE_START_PROPERTY_VALUE and tag == 'td':
            self._mode = MODE_SET_PROPERTY_VALUE
        elif tag == 'a' and len(attrs) > 0 and attrs[0][1].startswith('http://data.yandex.ru/i?ctype=1'):
            self._product[1].append(attrs[0][1])
                    
    
    def handle_data(self, data):
        if data:
            self._data = self._data + data

    def handle_entityref(self, name):
        if self.entitydefs.has_key(name): 
            self._data = self._data + self.entitydefs[name]
    
    def unknown_endtag(self, tag):
        print 'end tag'
        self._data = self._data.strip()
        if self._mode == MODE_SET_NAME and tag == 'h1':
            self._mode = MODE_DEFAULT
            self._product[0] = self._data
        elif self._mode == MODE_SET_GROUP_NAME and tag == 'td':
            self._current_group = []
            self._product[2].append([self._data, self._current_group])
            self._mode = MODE_START_PROPERTY_NAME  
        if self._mode == MODE_SET_PROPERTY_NAME and tag == 'td':
            self._mode = MODE_START_PROPERTY_VALUE
            self._current_prop = self._data
        if self._mode == MODE_SET_PROPERTY_VALUE and tag == 'td':
            self._mode = MODE_START_PROPERTY_NAME
            self._current_group.append([self._current_prop, self._data])
   
    def get_result(self):
        return self._product
       
if __name__ == "__main__":
    main()